{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bb034c64",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch import nn\n",
    "\n",
    "\n",
    "class CenteredLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, X):\n",
    "        return X - X.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8a8a046f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-2., -1.,  0.,  1.,  2.])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "layer = CenteredLayer()\n",
    "layer(torch.FloatTensor([1, 2, 3, 4, 5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "af312300",
   "metadata": {},
   "outputs": [],
   "source": [
    "net = nn.Sequential(nn.Linear(8, 128), CenteredLayer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0bc0706f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.0039, 0.3687, 0.0028, 0.0732, 0.9620, 0.9979, 0.8640, 0.0370],\n",
      "        [0.9264, 0.2972, 0.6523, 0.0479, 0.2645, 0.9409, 0.5921, 0.0312],\n",
      "        [0.1310, 0.9148, 0.9556, 0.9120, 0.5543, 0.6119, 0.6157, 0.8316],\n",
      "        [0.8519, 0.6676, 0.0594, 0.6002, 0.8179, 0.0587, 0.1391, 0.0103]])\n",
      "tensor([[ 0.0046,  0.3137,  0.2670, -0.0032,  0.6964,  0.5431, -0.2769,  0.2274,\n",
      "          0.3958, -0.8877, -0.4941, -0.2272,  0.1366, -0.1832, -0.5324, -0.2925,\n",
      "          0.0183,  0.1781, -0.6002,  0.6025, -0.0284,  0.3739,  0.3546,  0.2180,\n",
      "          0.0029, -0.3556, -0.2963,  1.0278,  0.6686,  0.5479, -0.2911,  0.1092,\n",
      "         -0.1370, -0.0611,  0.1367,  0.3503, -0.6357, -0.2158, -0.0442, -0.0485,\n",
      "          0.1411,  0.2767,  0.3735,  0.1582, -0.1800,  0.4523, -0.8247,  0.1063,\n",
      "         -0.0096, -0.3922,  0.5199,  0.0819, -0.2344,  0.2864, -0.0104,  0.2192,\n",
      "         -0.6871,  0.3839,  0.3830,  0.0141, -0.1448,  0.3223,  0.5096,  0.5378,\n",
      "          0.6204,  0.7005,  0.0270, -0.2696,  0.8919,  1.0859,  0.2950, -0.0547,\n",
      "         -0.0992, -0.0134, -0.5396, -0.2038, -0.4314, -0.2724,  0.1648, -0.6389,\n",
      "         -0.3692,  0.1550,  0.3363,  0.5460,  0.3490,  0.2048, -0.2558,  0.1662,\n",
      "         -0.3872, -0.2566,  0.6914,  0.3698, -0.1385, -0.0986,  0.1108,  0.1353,\n",
      "         -0.4711, -0.0169, -0.2510,  0.1720, -0.0176, -0.7181, -0.2546, -0.2449,\n",
      "          0.1370,  0.0323,  0.7355, -0.7193,  0.1286, -0.6625, -0.5810,  0.0356,\n",
      "          0.4579, -0.1037,  0.0196, -0.1034, -0.2004,  0.5094, -0.0215,  0.5014,\n",
      "          0.4700, -0.2444, -0.3364,  0.0742, -0.2865, -0.9271,  0.2736,  0.1793],\n",
      "        [ 0.0966, -0.1252, -0.0859,  0.2036,  0.4995,  0.3043, -0.6881,  0.0158,\n",
      "          0.0700, -0.7052, -0.0790, -0.5157,  0.1202, -0.4662, -0.0707, -0.7048,\n",
      "         -0.2483,  0.1534, -0.3595,  0.8092,  0.0876,  0.2014, -0.0775,  0.0118,\n",
      "         -0.1297, -0.1081, -0.2832,  0.5934,  0.4725,  0.2951, -0.3810,  0.2474,\n",
      "          0.3570, -0.4791, -0.0776,  0.8249,  0.0041, -0.1215, -0.3031, -0.2653,\n",
      "          0.3570,  0.1052,  0.2069, -0.5539, -0.0534,  0.1835, -0.5284,  0.3774,\n",
      "         -0.2587, -0.3426, -0.1131, -0.1058, -0.3995,  0.1683, -0.2520,  0.5216,\n",
      "         -0.1420,  0.4866,  0.5679,  0.0607, -0.1478, -0.1318,  0.2734,  0.2123,\n",
      "          0.6020,  0.0656, -0.1036,  0.1194,  0.6523,  0.7108,  0.1508,  0.0276,\n",
      "         -0.4344, -0.2098, -0.0458,  0.0099, -0.1376, -0.2540,  0.2580, -0.5138,\n",
      "          0.0164,  0.2957,  0.4437,  0.7730,  0.1680, -0.2817, -0.8214, -0.2620,\n",
      "         -0.2342,  0.1096,  0.3213,  0.3935, -0.5800, -0.4111,  0.1692,  0.1165,\n",
      "         -0.1630,  0.3721, -0.1723,  0.6164,  0.2160, -0.6594, -0.0933, -0.2044,\n",
      "          0.0994, -0.3007,  0.7121, -0.6798,  0.1791, -0.9286, -0.5350,  0.0357,\n",
      "          0.2873, -0.4859, -0.3841, -0.1665, -0.2260,  0.4497, -0.4622,  0.8143,\n",
      "          0.5712, -0.3514, -0.0826,  0.1919, -0.0824, -0.6337,  0.3789,  0.4978],\n",
      "        [ 0.0755, -0.0039, -0.0514,  0.4805,  0.6109,  0.0051, -0.1408,  0.0309,\n",
      "         -0.1271, -0.9465, -0.0130, -0.3159,  0.4485, -0.2064, -0.1253, -0.6410,\n",
      "         -0.3898,  0.0452,  0.3728,  0.7525,  0.2848,  0.7530,  0.4166, -0.1599,\n",
      "         -0.2787, -0.0121, -0.1089,  1.0227,  0.2625,  0.7504,  0.1039,  0.4505,\n",
      "          0.1940, -0.5373,  0.3735,  0.8063, -0.6794, -0.5374, -0.1703,  0.1250,\n",
      "          0.2486, -0.0293,  0.4416,  0.1724, -0.3357,  0.8508, -0.8186,  0.0730,\n",
      "         -0.1852, -0.3182, -0.2377, -0.3549, -0.6809,  0.2514,  0.1945,  0.5424,\n",
      "         -0.3815,  0.2503,  0.3142,  0.1541, -0.5314,  0.6282,  0.1416,  0.4943,\n",
      "          0.4202,  0.5706, -0.0028, -0.0175,  0.2857,  0.7782,  0.2942, -0.4528,\n",
      "         -0.3128, -0.9832, -0.4857, -0.2020, -0.4759, -0.3504,  0.0187, -0.6665,\n",
      "         -0.0112, -0.0917,  0.2150,  0.8285,  0.1028, -0.3172, -0.8537,  0.2329,\n",
      "         -0.3506,  0.1543,  0.5871,  0.6785, -0.8747, -0.7015, -0.4720,  0.0833,\n",
      "         -0.4557,  0.2994, -0.4130,  0.5665, -0.0306, -0.5840, -0.0233, -1.0363,\n",
      "          0.0183,  0.2013,  1.1748, -0.4549,  0.1419, -1.2896, -0.6415,  0.2239,\n",
      "          0.1718, -0.2146,  0.1718,  0.3094, -0.5260,  0.6657, -0.0885,  1.0799,\n",
      "          0.3319, -0.5714, -0.6867, -0.2262,  0.2476, -1.1602,  0.3736,  0.3049],\n",
      "        [ 0.2722, -0.3215, -0.3149,  0.3887,  0.3271, -0.1717,  0.0470,  0.3093,\n",
      "         -0.1023, -0.5188, -0.0460, -0.1375, -0.1585,  0.1009, -0.1732, -0.5501,\n",
      "         -0.1601,  0.1650, -0.1009,  0.6226, -0.1040,  0.5496,  0.1539,  0.1376,\n",
      "         -0.7877,  0.0852, -0.5631,  0.3393,  0.4818, -0.0232, -0.3636, -0.0314,\n",
      "          0.1367, -0.1528,  0.3574,  0.4723, -0.6032, -0.0715, -0.6662, -0.2697,\n",
      "          0.2159, -0.0653,  0.3601,  0.2585, -0.1459,  0.0768, -0.2884,  0.1449,\n",
      "          0.1198, -0.3420, -0.2363,  0.1295, -0.5854,  0.4552, -0.3711,  0.2383,\n",
      "         -0.1335,  0.2390, -0.0450,  0.2085, -0.3051,  0.2469,  0.2167, -0.0404,\n",
      "          0.8951,  0.2332,  0.2897,  0.0417,  0.4198,  0.4734, -0.1216, -0.2164,\n",
      "         -0.2779, -0.0296, -0.2250, -0.3358, -0.2104, -0.3904,  0.3528, -0.3373,\n",
      "          0.1254, -0.2674,  0.1385,  0.6469,  0.4284,  0.0053, -0.5896, -0.0910,\n",
      "         -0.2815, -0.0617,  0.0390,  0.3041, -0.2815, -0.2075,  0.1553, -0.0581,\n",
      "         -0.3142,  0.1484, -0.5986,  0.0609, -0.4158, -0.2929, -0.1696, -0.3367,\n",
      "         -0.1206,  0.0678,  0.3585, -0.3684, -0.0526, -0.5455, -0.2479,  0.1758,\n",
      "          0.6754, -0.0151,  0.0944,  0.2895, -0.5948,  0.4537, -0.5755,  0.3722,\n",
      "          0.1113, -0.3165, -0.7232,  0.2365,  0.0915, -0.5111,  0.1147,  0.4222]],\n",
      "       grad_fn=<SubBackward0>)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(0., grad_fn=<MeanBackward0>)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = torch.rand(4, 8)\n",
    "print(X)\n",
    "Y = net(X)\n",
    "print(Y)\n",
    "Y.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "7dc70828",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "print(X.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "07bab83e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MyLinear(nn.Module):\n",
    "    def __init__(self, in_units, units):\n",
    "        super().__init__()\n",
    "        self.weight = nn.Parameter(torch.randn(in_units, units))\n",
    "        self.bias = nn.Parameter(torch.randn(units,))\n",
    "    def forward(self, X):\n",
    "        linear = torch.matmul(X, self.weight.data) + self.bias.data\n",
    "        return F.relu(linear)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2b2e84bc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[ 1.3273, -1.0866,  2.1523],\n",
       "        [ 0.0989, -0.2853, -0.0874],\n",
       "        [-0.7482, -0.0123,  0.9957],\n",
       "        [-2.0440,  0.8397,  0.8877],\n",
       "        [-0.7472,  0.5242,  0.3697]], requires_grad=True)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear = MyLinear(5, 3)\n",
    "linear.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8371999a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.0000, 0.0000, 1.0491],\n",
       "        [0.0000, 0.0000, 1.6546]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "linear(torch.rand(2, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "894f1d31",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[6.8372],\n",
       "        [0.0000]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net = nn.Sequential(MyLinear(64, 8), MyLinear(8, 1))\n",
    "net(torch.rand(2, 64))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d7e42773",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
